import json
import time

import jsonpath
import scrapy


class CtSpider(scrapy.Spider):
    name = 'ct'
    allowed_domains = ['chouti.com']
    # afterTime发布的时间  _= 获取的时间
    time = time.time()
    start_urls = ['https://dig.chouti.com/link/hot?_=%s' % time]
    page = 1

    def parse(self, response):
        json_dict = json.loads(response.text)
        json_count = jsonpath.jsonpath(json_dict, "$..data")
        # print(json_count)
        # time_into_pool =jsonpath.jsonpath(json_count, "$..time_into_pool")
        next_page = json_dict.get("data")[24].get("time_into_pool")
        print(next_page)
        for json_temp in json_count:
            for json_str in json_temp:
                nick_dict = jsonpath.jsonpath(json_str, "$..submitted_user")
                for nick in nick_dict:
                    yield {
                        "type": "info",
                        "id": json_str['id'],
                        'title': json_str["title"],
                        "originalUrl": json_str["originalUrl"],
                        "nick": nick['nick'],
                        "commentsCount": json_str['comments_count'],
                        'ups': json_str['ups']
                    }
                    # id是新闻的id
                    comments_url = 'https://dig.chouti.com/comments/show?id=%s&sort=1&_=1606964143287' % (
                        json_str['id'])
                    yield scrapy.Request(url=comments_url, callback=self.comments, cb_kwargs={"id": json_str['id']})

        if self.page <= 3:
            self.page += 1
            url = 'https://dig.chouti.com/link/hot?afterTime=%s' % next_page
            yield scrapy.Request(url=url, callback=self.parse)

    def comments(self, response, id):
        # print("---评论----")
        comment_dict = json.loads(response.text)
        comment_count = jsonpath.jsonpath(comment_dict, "$..commentList")
        # print(comment_count)
        if comment_count:
            for comment_temp in comment_count:
                for comment_str in comment_temp:
                    if comment_str['content']:
                        content = comment_str['content']
                    else:
                        content = ""
                    yield {
                        "type": "update",
                        "id": id,
                        "content": content
                    }
